package med.extract;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;

import med.utils.Reader;

import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Extractor {

	private static final String SPLIT_DELIM = "\\s+";

	private final Logger log = Logger.getLogger(Extractor.class);

	private Set<String> visitedURLs = new HashSet<String>();

	private boolean useStemmer;
	
	public static void main(String[] args) {
		Extractor extr = new Extractor();
		
		final String fileName = args[0];
		final String url = args[1];
		
		File file = new File(fileName);
		try {
			extr.extractToFile(file, new URL(url));
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	}

	private void extractToFile(File file, URL url) throws IOException {
		BufferedWriter bw = new BufferedWriter(new FileWriter(file));
		List<String> text = getText(url, 1, false);
		for(String token : text) {
			bw.write(token);
			bw.write(" ");
		}
		bw.close();
	}

	public List<String> getText(URL rootURL, int maxDepth,
			boolean useStemmer) throws IOException {
		if (maxDepth < 0) {
			throw new IllegalArgumentException();
		}
		this.useStemmer = useStemmer;
		return extract(rootURL, maxDepth);
	}

	private List<String> extract(URL parentURL, int maxDepth)
			throws IOException {

		if (parentURL == null || visited(parentURL) || maxDepth < 0) {
			return Collections.emptyList();
		}

		
		Document doc = Jsoup.connect(parentURL.toString()).get();
		visit(parentURL);

		List<String> result = new LinkedList<String>();

		// extract text from page
		String[] text = doc.body().text().split(SPLIT_DELIM);
		for (String token : text) {
			token = Reader.process(token, useStemmer);
			if (!Reader.isSkippable(token)) {
				result.add(token);
			}
		}

		// extract text from subpages
		Elements links = doc.select("a[href]");
		for (Element el : links) {
			URL childURL = tryCreateURL(parentURL, getLinkFromElement(el));
			if (equalHost(parentURL, childURL)) {
				try {
					result.addAll(extract(childURL, maxDepth - 1));
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
		}

		return result;
	}
	
	private boolean visited(URL url) {
		return visitedURLs.contains(url.getHost() + url.getPath());
	}

	private void visit(URL url) {
		log.debug("visiting " + url);
		visitedURLs.add(url.getHost() + url.getPath());
	}

	private static URL tryCreateURL(URL context, String spec) {
		URL result = null;
		try {
			result = new URL(context, spec);
		} catch (MalformedURLException ignore) {
		}
		return result;
	}

	private static String getLinkFromElement(Element el) {
		return el.attr("href");
	}

	private static boolean equalHost(URL first, URL second) {
		if (first == null || second == null) {
			return false;
		}
		return first.getHost().toLowerCase()
				.equals(second.getHost().toLowerCase());
	}
}
